{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "44037f84-0d8d-4511-be4e-7e7899a51016",
   "metadata": {},
   "source": [
    "# MiniCPM-LLama3-v2.5 Ollama实现\n",
    "\n",
    "> 本地环境采用Transformer加载，启动不了。 因此采用量化模型，Ollama本地实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1c9f9987-3700-4f93-a61c-d055802207d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "llama-index-embeddings-ollama                    0.1.2\n",
      "llama-index-llms-ollama                          0.1.2\n"
     ]
    }
   ],
   "source": [
    "!pip list | grep ollama"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cb460c7-f225-4a73-84eb-16ecf58bc2ba",
   "metadata": {},
   "source": [
    "## 使用Ollama本地部署模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0297832b-8ca2-46ca-b064-278990e5e475",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/laobao/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from llama_index.llms.ollama import Ollama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4bc4b8e-1b06-4d7d-b636-6e3f9db00eda",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = Ollama(\n",
    "    model=\"llama3\", \n",
    "    request_timeout=120.0\n",
    ")\n",
    "\n",
    "resp = llm.complete(\"什么是数字孪生?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b8086c51-8634-4e84-9a70-2a61c6038c19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# resp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c332161-098d-4418-8a78-efaa6cb0190a",
   "metadata": {},
   "source": [
    "### chat 方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "edd8fa34-4899-4cf5-88b9-997fdbce2797",
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core.llms import ChatMessage\n",
    "\n",
    "messages = [\n",
    "    ChatMessage(\n",
    "        role=\"system\", \n",
    "        content=\"你是数字化制造领域专家，精通数字孪生、PLM，数字化转型、工业互联网等\"\n",
    "    ),\n",
    "    ChatMessage(\n",
    "        role=\"user\", \n",
    "        content=\"什么是数字主线\"\n",
    "    ),\n",
    "]\n",
    "resp = llm.chat(messages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "18b63f83-d521-4751-9ebf-12231af6c9a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# resp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90153f39-0489-498a-ab57-600e3d1488ba",
   "metadata": {},
   "source": [
    "#### Stream方式的Chat\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "43568f49-3f98-4994-be76-493dad9527c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "messages = [\n",
    "    ChatMessage(\n",
    "        role=\"system\", \n",
    "        content=\"你是数字化制造领域专家，精通数字孪生、PLM，数字化转型、工业互联网等\"\n",
    "    ),\n",
    "    ChatMessage(\n",
    "        role=\"user\", \n",
    "        content=\"什么是数字阴影（Digital Shadow）\"\n",
    "    ),\n",
    "]\n",
    "resp = llm.stream_chat(messages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f6508e5e-e9fa-4d19-b28e-aaf39aaae45d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "“数字阴影”一词通常用于描述由个人或组织在网络上留下的一切痕迹和数据的总体集合。这些痕迹可能包括社交媒体账户、电子邮件记录、搜索历史、在线活动等。这句话可以理解为一个人/一个组织的整个网络足迹。\n",
      "“数字阴影”这个词也被用于描述人工智能技术中的一种概念，即通过对大量数据进行学习和建模，生成个体特征或行为模式，从而实现自动化决策或预测。这种方法类似于我们在现实生活中的观察和推理，但是在数字世界中是基于算法和大数据的。\n",
      "总之，“数字阴影”这个概念描述了一个人/一个组织在线活动所留下的痕迹，以及人工智能技术对个体特征或行为模式进行分析和预测的一种方法。"
     ]
    }
   ],
   "source": [
    "for r in resp:\n",
    "    print(r.delta, end=\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "579de33a-528a-4ea7-86e3-80e9da95c545",
   "metadata": {},
   "source": [
    "## 仿照官网实现\n",
    "\n",
    "```\n",
    "# test.py\n",
    "import torch\n",
    "from PIL import Image\n",
    "from modelscope import AutoModel, AutoTokenizer\n",
    "\n",
    "model = AutoModel.from_pretrained('openbmb/MiniCPM-Llama3-V-2_5', trust_remote_code=True, torch_dtype=torch.float16)\n",
    "model = model.to(device='cuda')\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-Llama3-V-2_5', trust_remote_code=True)\n",
    "model.eval()\n",
    "\n",
    "image = Image.open('xx.jpg').convert('RGB')\n",
    "question = 'What is in the image?'\n",
    "msgs = [{'role': 'user', 'content': question}]\n",
    "\n",
    "res = model.chat(\n",
    "    image=image,\n",
    "    msgs=msgs,\n",
    "    tokenizer=tokenizer,\n",
    "    sampling=True,\n",
    "    temperature=0.7\n",
    ")\n",
    "print(res)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "68e61c1e-b5ae-4cee-ab3f-3551ebcfb5c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import numpy as np\n",
    "\n",
    "table_image = Image.open('../datasets/testcase-saic-table.jpg')\n",
    "# a = np.asarray(table_image)\n",
    "\n",
    "# im = Image.fromarray(a, mode=\"RGB\")\n",
    "# image1 = Image.fromarray('../datasets/testcase-saic-table.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e51805c1-feea-48c6-8dac-f8a35838bf74",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_vision = Ollama(\n",
    "    model=\"minicpm-v2.5\", \n",
    "    request_timeout=120.0\n",
    ")\n",
    "\n",
    "\n",
    "messages = [\n",
    "    ChatMessage(\n",
    "        role=\"system\", \n",
    "        content=\"你是数字化制造领域专家，精通数字孪生、PLM，数字化转型、工业互联网等\"\n",
    "    ),\n",
    "    ChatMessage(\n",
    "        role=\"user\", \n",
    "        content=\"What is in the image?\"\n",
    "    ),\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e2cdcb22-6527-4803-9d2e-b26ca4f37a1a",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Object of type JpegImageFile is not JSON serializable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[39], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# question = 'What is in the image?'\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;66;03m# msgs = [{'role': 'user', 'content': question}]\u001b[39;00m\n\u001b[0;32m----> 4\u001b[0m res \u001b[38;5;241m=\u001b[39m \u001b[43mllm_vision\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchat\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m    \u001b[49m\u001b[43mimage\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mtable_image\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m    \u001b[49m\u001b[43msampling\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtemperature\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0.7\u001b[39;49m\n\u001b[1;32m      9\u001b[0m \u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/llama_index/core/llms/callbacks.py:144\u001b[0m, in \u001b[0;36mllm_chat_callback.<locals>.wrap.<locals>.wrapped_llm_chat\u001b[0;34m(_self, messages, **kwargs)\u001b[0m\n\u001b[1;32m    128\u001b[0m dispatcher\u001b[38;5;241m.\u001b[39mevent(\n\u001b[1;32m    129\u001b[0m     LLMChatStartEvent(\n\u001b[1;32m    130\u001b[0m         model_dict\u001b[38;5;241m=\u001b[39m_self\u001b[38;5;241m.\u001b[39mto_dict(),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    134\u001b[0m     )\n\u001b[1;32m    135\u001b[0m )\n\u001b[1;32m    136\u001b[0m event_id \u001b[38;5;241m=\u001b[39m callback_manager\u001b[38;5;241m.\u001b[39mon_event_start(\n\u001b[1;32m    137\u001b[0m     CBEventType\u001b[38;5;241m.\u001b[39mLLM,\n\u001b[1;32m    138\u001b[0m     payload\u001b[38;5;241m=\u001b[39m{\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    142\u001b[0m     },\n\u001b[1;32m    143\u001b[0m )\n\u001b[0;32m--> 144\u001b[0m f_return_val \u001b[38;5;241m=\u001b[39m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_self\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    146\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(f_return_val, Generator):\n\u001b[1;32m    147\u001b[0m     \u001b[38;5;66;03m# intercept the generator and add a callback to the end\u001b[39;00m\n\u001b[1;32m    148\u001b[0m     \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapped_gen\u001b[39m() \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m ChatResponseGen:\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/llama_index/llms/ollama/base.py:101\u001b[0m, in \u001b[0;36mOllama.chat\u001b[0;34m(self, messages, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m payload \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m     86\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel,\n\u001b[1;32m     87\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     97\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m     98\u001b[0m }\n\u001b[1;32m    100\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m httpx\u001b[38;5;241m.\u001b[39mClient(timeout\u001b[38;5;241m=\u001b[39mTimeout(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrequest_timeout)) \u001b[38;5;28;01mas\u001b[39;00m client:\n\u001b[0;32m--> 101\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    102\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbase_url\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m/api/chat\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    103\u001b[0m \u001b[43m        \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpayload\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    104\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    105\u001b[0m     response\u001b[38;5;241m.\u001b[39mraise_for_status()\n\u001b[1;32m    106\u001b[0m     raw \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mjson()\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/httpx/_client.py:1145\u001b[0m, in \u001b[0;36mClient.post\u001b[0;34m(self, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions)\u001b[0m\n\u001b[1;32m   1124\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(\n\u001b[1;32m   1125\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m   1126\u001b[0m     url: URLTypes,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1138\u001b[0m     extensions: RequestExtensions \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m   1139\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Response:\n\u001b[1;32m   1140\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   1141\u001b[0m \u001b[38;5;124;03m    Send a `POST` request.\u001b[39;00m\n\u001b[1;32m   1142\u001b[0m \n\u001b[1;32m   1143\u001b[0m \u001b[38;5;124;03m    **Parameters**: See `httpx.request`.\u001b[39;00m\n\u001b[1;32m   1144\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1145\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1146\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mPOST\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1147\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1148\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1149\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1150\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1151\u001b[0m \u001b[43m        \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1152\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1153\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1154\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcookies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcookies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1155\u001b[0m \u001b[43m        \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1156\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfollow_redirects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfollow_redirects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1157\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1158\u001b[0m \u001b[43m        \u001b[49m\u001b[43mextensions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextensions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1159\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/httpx/_client.py:814\u001b[0m, in \u001b[0;36mClient.request\u001b[0;34m(self, method, url, content, data, files, json, params, headers, cookies, auth, follow_redirects, timeout, extensions)\u001b[0m\n\u001b[1;32m    807\u001b[0m     message \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    808\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSetting per-request cookies=<...> is being deprecated, because \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    809\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mthe expected behaviour on cookie persistence is ambiguous. Set \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    810\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcookies directly on the client instance instead.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    811\u001b[0m     )\n\u001b[1;32m    812\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(message, \u001b[38;5;167;01mDeprecationWarning\u001b[39;00m)\n\u001b[0;32m--> 814\u001b[0m request \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuild_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    815\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    816\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    817\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcontent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    818\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    819\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    820\u001b[0m \u001b[43m    \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    821\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    822\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    823\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcookies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcookies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    824\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    825\u001b[0m \u001b[43m    \u001b[49m\u001b[43mextensions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextensions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    826\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    827\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msend(request, auth\u001b[38;5;241m=\u001b[39mauth, follow_redirects\u001b[38;5;241m=\u001b[39mfollow_redirects)\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/httpx/_client.py:357\u001b[0m, in \u001b[0;36mBaseClient.build_request\u001b[0;34m(self, method, url, content, data, files, json, params, headers, cookies, timeout, extensions)\u001b[0m\n\u001b[1;32m    351\u001b[0m     timeout \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m    352\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtimeout\n\u001b[1;32m    353\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(timeout, UseClientDefault)\n\u001b[1;32m    354\u001b[0m         \u001b[38;5;28;01melse\u001b[39;00m Timeout(timeout)\n\u001b[1;32m    355\u001b[0m     )\n\u001b[1;32m    356\u001b[0m     extensions \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mextensions, timeout\u001b[38;5;241m=\u001b[39mtimeout\u001b[38;5;241m.\u001b[39mas_dict())\n\u001b[0;32m--> 357\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mRequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    358\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    359\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    360\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcontent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    361\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    362\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    363\u001b[0m \u001b[43m    \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    364\u001b[0m \u001b[43m    \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    365\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    366\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcookies\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcookies\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    367\u001b[0m \u001b[43m    \u001b[49m\u001b[43mextensions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextensions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    368\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/httpx/_models.py:340\u001b[0m, in \u001b[0;36mRequest.__init__\u001b[0;34m(self, method, url, params, headers, cookies, content, data, files, json, stream, extensions)\u001b[0m\n\u001b[1;32m    338\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stream \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    339\u001b[0m     content_type: \u001b[38;5;28mstr\u001b[39m \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheaders\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent-type\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 340\u001b[0m     headers, stream \u001b[38;5;241m=\u001b[39m \u001b[43mencode_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    341\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcontent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    342\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    343\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    344\u001b[0m \u001b[43m        \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    345\u001b[0m \u001b[43m        \u001b[49m\u001b[43mboundary\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mget_multipart_boundary_from_content_type\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    346\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcontent_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontent_type\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    347\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mcontent_type\u001b[49m\n\u001b[1;32m    348\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\n\u001b[1;32m    349\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    350\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    351\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare(headers)\n\u001b[1;32m    352\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstream \u001b[38;5;241m=\u001b[39m stream\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/httpx/_content.py:212\u001b[0m, in \u001b[0;36mencode_request\u001b[0;34m(content, data, files, json, boundary)\u001b[0m\n\u001b[1;32m    210\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m encode_urlencoded_data(data)\n\u001b[1;32m    211\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m json \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 212\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mencode_json\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    214\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m {}, ByteStream(\u001b[38;5;124mb\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/site-packages/httpx/_content.py:175\u001b[0m, in \u001b[0;36mencode_json\u001b[0;34m(json)\u001b[0m\n\u001b[1;32m    174\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mencode_json\u001b[39m(json: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m], ByteStream]:\n\u001b[0;32m--> 175\u001b[0m     body \u001b[38;5;241m=\u001b[39m \u001b[43mjson_dumps\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjson\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mencode(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    176\u001b[0m     content_length \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mlen\u001b[39m(body))\n\u001b[1;32m    177\u001b[0m     content_type \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/json\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/json/__init__.py:231\u001b[0m, in \u001b[0;36mdumps\u001b[0;34m(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)\u001b[0m\n\u001b[1;32m    226\u001b[0m \u001b[38;5;66;03m# cached encoder\u001b[39;00m\n\u001b[1;32m    227\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;129;01mnot\u001b[39;00m skipkeys \u001b[38;5;129;01mand\u001b[39;00m ensure_ascii \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    228\u001b[0m     check_circular \u001b[38;5;129;01mand\u001b[39;00m allow_nan \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    229\u001b[0m     \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m indent \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m separators \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    230\u001b[0m     default \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m sort_keys \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 231\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_encoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    232\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    233\u001b[0m     \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONEncoder\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/json/encoder.py:199\u001b[0m, in \u001b[0;36mJSONEncoder.encode\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    195\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m encode_basestring(o)\n\u001b[1;32m    196\u001b[0m \u001b[38;5;66;03m# This doesn't pass the iterator directly to ''.join() because the\u001b[39;00m\n\u001b[1;32m    197\u001b[0m \u001b[38;5;66;03m# exceptions aren't as detailed.  The list call should be roughly\u001b[39;00m\n\u001b[1;32m    198\u001b[0m \u001b[38;5;66;03m# equivalent to the PySequence_Fast that ''.join() would do.\u001b[39;00m\n\u001b[0;32m--> 199\u001b[0m chunks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miterencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_one_shot\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m    200\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(chunks, (\u001b[38;5;28mlist\u001b[39m, \u001b[38;5;28mtuple\u001b[39m)):\n\u001b[1;32m    201\u001b[0m     chunks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(chunks)\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/json/encoder.py:257\u001b[0m, in \u001b[0;36mJSONEncoder.iterencode\u001b[0;34m(self, o, _one_shot)\u001b[0m\n\u001b[1;32m    252\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    253\u001b[0m     _iterencode \u001b[38;5;241m=\u001b[39m _make_iterencode(\n\u001b[1;32m    254\u001b[0m         markers, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdefault, _encoder, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mindent, floatstr,\n\u001b[1;32m    255\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkey_separator, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mitem_separator, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msort_keys,\n\u001b[1;32m    256\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mskipkeys, _one_shot)\n\u001b[0;32m--> 257\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_iterencode\u001b[49m\u001b[43m(\u001b[49m\u001b[43mo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/opt/anaconda3/envs/llama-index/lib/python3.10/json/encoder.py:179\u001b[0m, in \u001b[0;36mJSONEncoder.default\u001b[0;34m(self, o)\u001b[0m\n\u001b[1;32m    160\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdefault\u001b[39m(\u001b[38;5;28mself\u001b[39m, o):\n\u001b[1;32m    161\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Implement this method in a subclass such that it returns\u001b[39;00m\n\u001b[1;32m    162\u001b[0m \u001b[38;5;124;03m    a serializable object for ``o``, or calls the base implementation\u001b[39;00m\n\u001b[1;32m    163\u001b[0m \u001b[38;5;124;03m    (to raise a ``TypeError``).\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    177\u001b[0m \n\u001b[1;32m    178\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 179\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mObject of type \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mo\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    180\u001b[0m                     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mis not JSON serializable\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "\u001b[0;31mTypeError\u001b[0m: Object of type JpegImageFile is not JSON serializable"
     ]
    }
   ],
   "source": [
    "# question = 'What is in the image?'\n",
    "# msgs = [{'role': 'user', 'content': question}]\n",
    "\n",
    "res = llm_vision.chat(\n",
    "    image = table_image,\n",
    "    messages = messages,\n",
    "    sampling = True,\n",
    "    temperature = 0.7\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "857cb7e0-97ca-4063-9621-4d95c3fa8939",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
